--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
tests/backend/test_benchmark_gate.py e5756d7ab065475a18acdcafc008a767fcff87e0 (e5756d7a) Text, 12.08 KB
T8b949e# SPDX-License-Identifier: 0BSD
Tff7b72import T7ee787json
Tff7b72import T7ee787os
Tff7b72import T7ee787tempfile
Tff7b72import T7ee787unittest
Tff7b72from T7ee787testsT7ee787.T7ee787backendT7ee787.T7ee787benchmarking_utils Tff7b72import Tb4b4b4(
Te6edf3BenchmarkResultTb4b4b4,
Te6edf3adaptive_alert_ratioTb4b4b4,
Te6edf3aggregate_run_resultsTb4b4b4,
Te6edf3coefficient_of_variationTb4b4b4,
Te6edf3medianTb4b4b4,
Te6edf3median_abs_deviationTb4b4b4,
Te6edf3parse_extra_statsTb4b4b4,
Te6edf3should_alert_regressionTb4b4b4,
Tb4b4b4)
Tff7b72from T7ee787testsT7ee787.T7ee787backendT7ee787.T7ee787compare_benchmarks Tff7b72import Tb4b4b4(
Te6edf3compareTb4b4b4,
Te6edf3update_baselineTb4b4b4,
Tb4b4b4)
Tff7b72from T7ee787testsT7ee787.T7ee787backendT7ee787.T7ee787compare_benchmarks Tff7b72import Tb4b4b4(
Te6edf3main Tff7b72as Te6edf3compare_mainTb4b4b4,
Tb4b4b4)
Tff7b72class T56d364TestBenchmarkStatsTb4b4b4(Te6edf3unittestTff7b72.Td2a8ffTestCaseTb4b4b4)Tb4b4b4:
Tff7b72def Td2a8fftest_median_odd_evenTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3medianTb4b4b4(Tb4b4b4[T79c0ff3Tb4b4b4, T79c0ff1Tb4b4b4, T79c0ff2Tb4b4b4]Tb4b4b4)Tb4b4b4, T79c0ff2Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3medianTb4b4b4(Tb4b4b4[T79c0ff4Tb4b4b4, T79c0ff1Tb4b4b4, T79c0ff3Tb4b4b4, T79c0ff2Tb4b4b4]Tb4b4b4)Tb4b4b4, T79c0ff2.5Tb4b4b4)
Tff7b72def Td2a8fftest_mad_and_cvTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Te6edf3samples Tff7b72= Tb4b4b4[T79c0ff10.0Tb4b4b4, T79c0ff10.0Tb4b4b4, T79c0ff10.0Tb4b4b4, T79c0ff12.0Tb4b4b4, T79c0ff8.0Tb4b4b4]
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3median_abs_deviationTb4b4b4(Te6edf3samplesTb4b4b4)Tb4b4b4, T79c0ff0.0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertGreaterTb4b4b4(Te6edf3coefficient_of_variationTb4b4b4(Te6edf3samplesTb4b4b4)Tb4b4b4, T79c0ff0.0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3coefficient_of_variationTb4b4b4(Tb4b4b4[T79c0ff5.0Tb4b4b4]Tb4b4b4)Tb4b4b4, T79c0ff0.0Tb4b4b4)
Tff7b72def Td2a8fftest_aggregate_run_results_median_of_mediansTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Te6edf3r1 Tff7b72= Te6edf3BenchmarkResultTb4b4b4(Ta5d6ff"Ta5d6ffATa5d6ff"Tb4b4b4, T79c0ff10.0Tb4b4b4, T79c0ff0.1Tb4b4b4, Te6edf3samples_msTff7b72=Tb4b4b4[T79c0ff9Tb4b4b4, T79c0ff10Tb4b4b4, T79c0ff11Tb4b4b4]Tb4b4b4)
Te6edf3r2 Tff7b72= Te6edf3BenchmarkResultTb4b4b4(Ta5d6ff"Ta5d6ffATa5d6ff"Tb4b4b4, T79c0ff20.0Tb4b4b4, T79c0ff0.2Tb4b4b4, Te6edf3samples_msTff7b72=Tb4b4b4[T79c0ff19Tb4b4b4, T79c0ff20Tb4b4b4, T79c0ff21Tb4b4b4]Tb4b4b4)
Te6edf3r3 Tff7b72= Te6edf3BenchmarkResultTb4b4b4(Ta5d6ff"Ta5d6ffATa5d6ff"Tb4b4b4, T79c0ff12.0Tb4b4b4, T79c0ff0.15Tb4b4b4, Te6edf3samples_msTff7b72=Tb4b4b4[T79c0ff11Tb4b4b4, T79c0ff12Tb4b4b4, T79c0ff13Tb4b4b4]Tb4b4b4)
Te6edf3b1 Tff7b72= Te6edf3BenchmarkResultTb4b4b4(Ta5d6ff"Ta5d6ffBTa5d6ff"Tb4b4b4, T79c0ff1.0Tb4b4b4, T79c0ff0.0Tb4b4b4, Te6edf3samples_msTff7b72=Tb4b4b4[T79c0ff1.0Tb4b4b4]Tb4b4b4)
Te6edf3b2 Tff7b72= Te6edf3BenchmarkResultTb4b4b4(Ta5d6ff"Ta5d6ffBTa5d6ff"Tb4b4b4, T79c0ff3.0Tb4b4b4, T79c0ff0.0Tb4b4b4, Te6edf3samples_msTff7b72=Tb4b4b4[T79c0ff3.0Tb4b4b4]Tb4b4b4)
Te6edf3b3 Tff7b72= Te6edf3BenchmarkResultTb4b4b4(Ta5d6ff"Ta5d6ffBTa5d6ff"Tb4b4b4, T79c0ff2.0Tb4b4b4, T79c0ff0.0Tb4b4b4, Te6edf3samples_msTff7b72=Tb4b4b4[T79c0ff2.0Tb4b4b4]Tb4b4b4)
Te6edf3out Tff7b72= Te6edf3aggregate_run_resultsTb4b4b4(Tb4b4b4[Tb4b4b4[Te6edf3r1Tb4b4b4, Te6edf3b1Tb4b4b4]Tb4b4b4, Tb4b4b4[Te6edf3r2Tb4b4b4, Te6edf3b2Tb4b4b4]Tb4b4b4, Tb4b4b4[Te6edf3r3Tb4b4b4, Te6edf3b3Tb4b4b4]Tb4b4b4]Tb4b4b4)
Te6edf3by_name Tff7b72= Tb4b4b4{Te6edf3rTff7b72.Td2a8ffnameTb4b4b4: Te6edf3r Tff7b72for Te6edf3r Tff7b72in Te6edf3outTb4b4b4}
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3by_nameTb4b4b4[Ta5d6ff"Ta5d6ffATa5d6ff"Tb4b4b4]Tff7b72.Td2a8ffduration_msTb4b4b4, T79c0ff12.0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3by_nameTb4b4b4[Ta5d6ff"Ta5d6ffBTa5d6ff"Tb4b4b4]Tff7b72.Td2a8ffduration_msTb4b4b4, T79c0ff2.0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Tffa657lenTb4b4b4(Te6edf3by_nameTb4b4b4[Ta5d6ff"Ta5d6ffATa5d6ff"Tb4b4b4]Tff7b72.Td2a8ffsamples_msTb4b4b4)Tb4b4b4, T79c0ff9Tb4b4b4)
Tff7b72def Td2a8fftest_adaptive_alert_ratio_widens_for_tiny_baselinesTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3adaptive_alert_ratioTb4b4b4(T79c0ff0.2Tb4b4b4)Tb4b4b4, T79c0ff4.0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3adaptive_alert_ratioTb4b4b4(T79c0ff2.0Tb4b4b4)Tb4b4b4, T79c0ff2.5Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3adaptive_alert_ratioTb4b4b4(T79c0ff10.0Tb4b4b4)Tb4b4b4, T79c0ff2.0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3adaptive_alert_ratioTb4b4b4(T79c0ff50.0Tb4b4b4)Tb4b4b4, T79c0ff1.5Tb4b4b4)
Tff7b72def Td2a8fftest_should_alert_skips_noise_floorTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Te6edf3alertTb4b4b4, Te6edf3reason Tff7b72= Te6edf3should_alert_regressionTb4b4b4(T79c0ff0.4Tb4b4b4, T79c0ff0.15Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertFalseTb4b4b4(Te6edf3alertTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertInTb4b4b4(Ta5d6ff"Ta5d6ffnoise floorTa5d6ff"Tb4b4b4, Te6edf3reasonTb4b4b4)
Tff7b72def Td2a8fftest_should_alert_skips_tiny_abs_deltaTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
T8b949e# 3x ratio but only +0.4 ms absolute — not actionable
Te6edf3alertTb4b4b4, Te6edf3reason Tff7b72= Te6edf3should_alert_regressionTb4b4b4(T79c0ff0.6Tb4b4b4, T79c0ff0.2Tb4b4b4, Te6edf3noise_floor_msTff7b72=T79c0ff0.1Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertFalseTb4b4b4(Te6edf3alertTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertInTb4b4b4(Ta5d6ff"Ta5d6ffabs deltaTa5d6ff"Tb4b4b4, Te6edf3reasonTb4b4b4)
Tff7b72def Td2a8fftest_should_alert_real_regressionTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
T8b949e# Large baseline, clear absolute and ratio regression
Te6edf3alertTb4b4b4, Te6edf3reason Tff7b72= Te6edf3should_alert_regressionTb4b4b4(T79c0ff40.0Tb4b4b4, T79c0ff10.0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertTrueTb4b4b4(Te6edf3alertTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertInTb4b4b4(Ta5d6ff"Ta5d6ffslowerTa5d6ff"Tb4b4b4, Te6edf3reasonTb4b4b4)
Tff7b72def Td2a8fftest_should_alert_within_adaptive_thresholdTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
T8b949e# 1.8x on a 10 ms baseline — adaptive threshold is 2.0x
Te6edf3alertTb4b4b4, Te6edf3reason Tff7b72= Te6edf3should_alert_regressionTb4b4b4(T79c0ff18.0Tb4b4b4, T79c0ff10.0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertFalseTb4b4b4(Te6edf3alertTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertInTb4b4b4(Ta5d6ff"Ta5d6ffwithin adaptiveTa5d6ff"Tb4b4b4, Te6edf3reasonTb4b4b4)
Tff7b72def Td2a8fftest_should_alert_widens_when_cv_highTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
T8b949e# Would alert at 2.1x on 10 ms baseline (threshold 2.0), but high CV
T8b949e# bumps threshold to 2.5 so this stays quiet.
Te6edf3alertTb4b4b4, Te6edf3_ Tff7b72= Te6edf3should_alert_regressionTb4b4b4(
T79c0ff21.0Tb4b4b4,
T79c0ff10.0Tb4b4b4,
Te6edf3current_cvTff7b72=T79c0ff0.5Tb4b4b4,
Te6edf3previous_cvTff7b72=T79c0ff0.1Tb4b4b4,
Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertFalseTb4b4b4(Te6edf3alertTb4b4b4)
Tff7b72def Td2a8fftest_parse_extra_statsTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Te6edf3stats Tff7b72= Te6edf3parse_extra_statsTb4b4b4(Ta5d6ff"Ta5d6ffmad=0.012 cv=0.150 runs=3 mem_delta_mb=0.10Ta5d6ff"Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertAlmostEqualTb4b4b4(Te6edf3statsTb4b4b4[Ta5d6ff"Ta5d6ffmadTa5d6ff"Tb4b4b4]Tb4b4b4, T79c0ff0.012Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertAlmostEqualTb4b4b4(Te6edf3statsTb4b4b4[Ta5d6ff"Ta5d6ffcvTa5d6ff"Tb4b4b4]Tb4b4b4, T79c0ff0.15Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertAlmostEqualTb4b4b4(Te6edf3statsTb4b4b4[Ta5d6ff"Ta5d6ffrunsTa5d6ff"Tb4b4b4]Tb4b4b4, T79c0ff3.0Tb4b4b4)
Tff7b72class T56d364TestCompareBenchmarksTb4b4b4(Te6edf3unittestTff7b72.Td2a8ffTestCaseTb4b4b4)Tb4b4b4:
Tff7b72def Td2a8ff_writeTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3pathTb4b4b4, Te6edf3entriesTb4b4b4)Tb4b4b4:
Tff7b72with Tffa657openTb4b4b4(Te6edf3pathTb4b4b4, Ta5d6ff"Ta5d6ffwTa5d6ff"Tb4b4b4, Te6edf3encodingTff7b72=Ta5d6ff"Ta5d6ffutf-8Ta5d6ff"Tb4b4b4) Tff7b72as Te6edf3fTb4b4b4:
Te6edf3jsonTff7b72.Td2a8ffdumpTb4b4b4(Te6edf3entriesTb4b4b4, Te6edf3fTb4b4b4)
Tff7b72def Td2a8fftest_compare_flags_real_regression_onlyTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72with Te6edf3tempfileTff7b72.Td2a8ffTemporaryDirectoryTb4b4b4(Tb4b4b4) Tff7b72as Te6edf3tmpTb4b4b4:
Te6edf3current Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffcurrent.jsonTa5d6ff"Tb4b4b4)
Te6edf3previous Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffprevious.jsonTa5d6ff"Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3currentTb4b4b4,
Tb4b4b4[
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffTiny NoiseTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff0.4Tb4b4b4,
Ta5d6ff"Ta5d6ffextraTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffcv=0.1Ta5d6ff"Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffReal SlowdownTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff40.0Tb4b4b4,
Ta5d6ff"Ta5d6ffextraTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffcv=0.05Ta5d6ff"Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffRatio NoiseTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff0.455Tb4b4b4,
Ta5d6ff"Ta5d6ffextraTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffcv=0.2Ta5d6ff"Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3previousTb4b4b4,
Tb4b4b4[
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffTiny NoiseTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff0.15Tb4b4b4,
Ta5d6ff"Ta5d6ffextraTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffcv=0.1Ta5d6ff"Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffReal SlowdownTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff10.0Tb4b4b4,
Ta5d6ff"Ta5d6ffextraTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffcv=0.05Ta5d6ff"Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffRatio NoiseTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff0.177Tb4b4b4,
Ta5d6ff"Ta5d6ffextraTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffcv=0.2Ta5d6ff"Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Te6edf3codeTb4b4b4, Te6edf3rows Tff7b72= Te6edf3compareTb4b4b4(
Te6edf3currentTb4b4b4,
Te6edf3previousTb4b4b4,
Te6edf3noise_floor_msTff7b72=T79c0ff0.5Tb4b4b4,
Te6edf3min_abs_delta_msTff7b72=T79c0ff1.5Tb4b4b4,
Tb4b4b4)
Te6edf3by_name Tff7b72= Tb4b4b4{Te6edf3rTb4b4b4[Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4]Tb4b4b4: Te6edf3r Tff7b72for Te6edf3r Tff7b72in Te6edf3rowsTb4b4b4}
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3codeTb4b4b4, T79c0ff1Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3by_nameTb4b4b4[Ta5d6ff"Ta5d6ffReal SlowdownTa5d6ff"Tb4b4b4]Tb4b4b4[Ta5d6ff"Ta5d6ffstatusTa5d6ff"Tb4b4b4]Tb4b4b4, Ta5d6ff"Ta5d6ffREGRESSIONTa5d6ff"Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3by_nameTb4b4b4[Ta5d6ff"Ta5d6ffTiny NoiseTa5d6ff"Tb4b4b4]Tb4b4b4[Ta5d6ff"Ta5d6ffstatusTa5d6ff"Tb4b4b4]Tb4b4b4, Ta5d6ff"Ta5d6ffnoiseTa5d6ff"Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3by_nameTb4b4b4[Ta5d6ff"Ta5d6ffRatio NoiseTa5d6ff"Tb4b4b4]Tb4b4b4[Ta5d6ff"Ta5d6ffstatusTa5d6ff"Tb4b4b4]Tb4b4b4, Ta5d6ff"Ta5d6ffnoiseTa5d6ff"Tb4b4b4)
Tff7b72def Td2a8fftest_compare_trim_announce_style_false_positiveTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
T8b949e"""The exact CI false-positive pattern: 0.177 -> 0.455 (2.57x)."""
Tff7b72with Te6edf3tempfileTff7b72.Td2a8ffTemporaryDirectoryTb4b4b4(Tb4b4b4) Tff7b72as Te6edf3tmpTb4b4b4:
Te6edf3current Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffcurrent.jsonTa5d6ff"Tb4b4b4)
Te6edf3previous Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffprevious.jsonTa5d6ff"Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3currentTb4b4b4,
Tb4b4b4[
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffTrim Announces for AspectTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff0.455Tb4b4b4,
Ta5d6ff"Ta5d6ffextraTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmad=0.05 cv=0.3 runs=1Ta5d6ff"Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3previousTb4b4b4,
Tb4b4b4[
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffTrim Announces for AspectTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff0.177Tb4b4b4,
Ta5d6ff"Ta5d6ffextraTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmad=0.02 cv=0.2 runs=1Ta5d6ff"Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Te6edf3codeTb4b4b4, Te6edf3rows Tff7b72= Te6edf3compareTb4b4b4(Te6edf3currentTb4b4b4, Te6edf3previousTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3codeTb4b4b4, T79c0ff0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3rowsTb4b4b4[T79c0ff0Tb4b4b4]Tb4b4b4[Ta5d6ff"Ta5d6ffstatusTa5d6ff"Tb4b4b4]Tb4b4b4, Ta5d6ff"Ta5d6ffnoiseTa5d6ff"Tb4b4b4)
Tff7b72def Td2a8fftest_update_baseline_and_reloadTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72with Te6edf3tempfileTff7b72.Td2a8ffTemporaryDirectoryTb4b4b4(Tb4b4b4) Tff7b72as Te6edf3tmpTb4b4b4:
Te6edf3current Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffcurrent.jsonTa5d6ff"Tb4b4b4)
Te6edf3baseline Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffcacheTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffbenchmark-data.jsonTa5d6ff"Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3currentTb4b4b4,
Tb4b4b4[Tb4b4b4{Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffConfig Get (50 keys)Ta5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff0.4Tb4b4b4}Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Te6edf3update_baselineTb4b4b4(Te6edf3currentTb4b4b4, Te6edf3baselineTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertTrueTb4b4b4(Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffisfileTb4b4b4(Te6edf3baselineTb4b4b4)Tb4b4b4)
Te6edf3codeTb4b4b4, Te6edf3rows Tff7b72= Te6edf3compareTb4b4b4(Te6edf3currentTb4b4b4, Te6edf3baselineTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3codeTb4b4b4, T79c0ff0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3rowsTb4b4b4[T79c0ff0Tb4b4b4]Tb4b4b4[Ta5d6ff"Ta5d6ffstatusTa5d6ff"Tb4b4b4]Tb4b4b4, Ta5d6ff"Ta5d6ffokTa5d6ff"Tb4b4b4)
Tff7b72def Td2a8fftest_cli_exits_nonzero_on_regressionTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72with Te6edf3tempfileTff7b72.Td2a8ffTemporaryDirectoryTb4b4b4(Tb4b4b4) Tff7b72as Te6edf3tmpTb4b4b4:
Te6edf3current Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffcurrent.jsonTa5d6ff"Tb4b4b4)
Te6edf3previous Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffprevious.jsonTa5d6ff"Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3currentTb4b4b4,
Tb4b4b4[Tb4b4b4{Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffCreate IdentityTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff200.0Tb4b4b4}Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3previousTb4b4b4,
Tb4b4b4[Tb4b4b4{Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffCreate IdentityTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff80.0Tb4b4b4}Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Te6edf3code Tff7b72= Te6edf3compare_mainTb4b4b4(
Tb4b4b4[
Ta5d6ff"Ta5d6ff--currentTa5d6ff"Tb4b4b4,
Te6edf3currentTb4b4b4,
Ta5d6ff"Ta5d6ff--previousTa5d6ff"Tb4b4b4,
Te6edf3previousTb4b4b4,
Ta5d6ff"Ta5d6ff--noise-floor-msTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ff0.5Ta5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ff--min-abs-delta-msTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ff1.5Ta5d6ff"Tb4b4b4,
Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3codeTb4b4b4, T79c0ff1Tb4b4b4)
Tff7b72def Td2a8fftest_full_suite_fails_when_required_bench_missingTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72with Te6edf3tempfileTff7b72.Td2a8ffTemporaryDirectoryTb4b4b4(Tb4b4b4) Tff7b72as Te6edf3tmpTb4b4b4:
Te6edf3current Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffcurrent.jsonTa5d6ff"Tb4b4b4)
Te6edf3previous Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffprevious.jsonTa5d6ff"Tb4b4b4)
T8b949e# Full-suite signal without the new required conversation benches.
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3previousTb4b4b4,
Tb4b4b4[
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffDatabase InitializationTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff20.0Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffGet 100 Conversations ListTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff5.0Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3currentTb4b4b4,
Tb4b4b4[
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffDatabase InitializationTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff21.0Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffGet 100 Conversations ListTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff5.1Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Te6edf3codeTb4b4b4, Te6edf3rows Tff7b72= Te6edf3compareTb4b4b4(Te6edf3currentTb4b4b4, Te6edf3previousTb4b4b4)
Te6edf3by_name Tff7b72= Tb4b4b4{Te6edf3rTb4b4b4[Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4]Tb4b4b4: Te6edf3r Tff7b72for Te6edf3r Tff7b72in Te6edf3rowsTb4b4b4}
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3codeTb4b4b4, T79c0ff1Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(
Te6edf3by_nameTb4b4b4[Ta5d6ff"Ta5d6ffGet Conversations Slim List (handler)Ta5d6ff"Tb4b4b4]Tb4b4b4[Ta5d6ff"Ta5d6ffstatusTa5d6ff"Tb4b4b4]Tb4b4b4,
Ta5d6ff"Ta5d6ffMISSINGTa5d6ff"Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8fftest_full_suite_fails_when_bench_removedTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72with Te6edf3tempfileTff7b72.Td2a8ffTemporaryDirectoryTb4b4b4(Tb4b4b4) Tff7b72as Te6edf3tmpTb4b4b4:
Te6edf3current Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffcurrent.jsonTa5d6ff"Tb4b4b4)
Te6edf3previous Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffprevious.jsonTa5d6ff"Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3previousTb4b4b4,
Tb4b4b4[
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffDatabase InitializationTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff20.0Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffExtra Optional BenchTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff1.0Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4]Tb4b4b4,
Tb4b4b4)
T8b949e# Current only has init: required benches missing + optional removed.
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3currentTb4b4b4,
Tb4b4b4[
Tb4b4b4{
Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffDatabase InitializationTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff20.0Tb4b4b4,
Tb4b4b4}Tb4b4b4,
Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Te6edf3codeTb4b4b4, Te6edf3rows Tff7b72= Te6edf3compareTb4b4b4(Te6edf3currentTb4b4b4, Te6edf3previousTb4b4b4)
Te6edf3by_name Tff7b72= Tb4b4b4{Te6edf3rTb4b4b4[Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4]Tb4b4b4: Te6edf3r Tff7b72for Te6edf3r Tff7b72in Te6edf3rowsTb4b4b4}
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3codeTb4b4b4, T79c0ff1Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3by_nameTb4b4b4[Ta5d6ff"Ta5d6ffExtra Optional BenchTa5d6ff"Tb4b4b4]Tb4b4b4[Ta5d6ff"Ta5d6ffstatusTa5d6ff"Tb4b4b4]Tb4b4b4, Ta5d6ff"Ta5d6ffREMOVEDTa5d6ff"Tb4b4b4)
Tff7b72def Td2a8fftest_toy_fixture_does_not_enforce_required_coverageTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72with Te6edf3tempfileTff7b72.Td2a8ffTemporaryDirectoryTb4b4b4(Tb4b4b4) Tff7b72as Te6edf3tmpTb4b4b4:
Te6edf3current Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffcurrent.jsonTa5d6ff"Tb4b4b4)
Te6edf3previous Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Te6edf3tmpTb4b4b4, Ta5d6ff"Ta5d6ffprevious.jsonTa5d6ff"Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3currentTb4b4b4,
Tb4b4b4[Tb4b4b4{Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffToy BenchTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff1.0Tb4b4b4}Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ff_writeTb4b4b4(
Te6edf3previousTb4b4b4,
Tb4b4b4[Tb4b4b4{Ta5d6ff"Ta5d6ffnameTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffToy BenchTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffunitTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmsTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: T79c0ff1.0Tb4b4b4}Tb4b4b4]Tb4b4b4,
Tb4b4b4)
Te6edf3codeTb4b4b4, Te6edf3rows Tff7b72= Te6edf3compareTb4b4b4(Te6edf3currentTb4b4b4, Te6edf3previousTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3codeTb4b4b4, T79c0ff0Tb4b4b4)
Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Te6edf3rowsTb4b4b4[T79c0ff0Tb4b4b4]Tb4b4b4[Ta5d6ff"Ta5d6ffstatusTa5d6ff"Tb4b4b4]Tb4b4b4, Ta5d6ff"Ta5d6ffokTa5d6ff"Tb4b4b4)
Tff7b72if Tff7b72__name__ Tff7b72== Ta5d6ff"Ta5d6ff__main__Ta5d6ff"Tb4b4b4:
Te6edf3unittestTff7b72.Td2a8ffmainTb4b4b4(Tb4b4b4)
──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────